{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "X34yQtydjOr1"
   },
   "source": [
    "### Initialization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "SrrUwLBnjOsl",
    "outputId": "23d231b0-8157-4fc2-e50b-5103dea5fd4c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TensorFlow 2.x selected.\n"
     ]
    }
   ],
   "source": [
    "# For Colab only!\n",
    "\n",
    "try:\n",
    "  # %tensorflow_version only exists in Colab.\n",
    "  %tensorflow_version 2.x\n",
    "except Exception:\n",
    "  pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "cell_style": "split",
    "colab": {},
    "colab_type": "code",
    "id": "yuq-2fGvjOs5"
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "cell_style": "split",
    "colab": {},
    "colab_type": "code",
    "id": "Q_PPqHnCjOtB"
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch.nn import functional as F\n",
    "from torchvision import datasets, transforms\n",
    "from torch import nn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "cell_style": "split",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 126
    },
    "colab_type": "code",
    "id": "Kg-NmTOzjOtI",
    "outputId": "087e2e25-1d35-4474-bd8b-92bbb2d0c492",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.1.0\n",
      "WARNING:tensorflow:From <ipython-input-3-2e82a26757ae>:2: is_gpu_available (from tensorflow.python.framework.test_util) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.config.list_physical_devices('GPU')` instead.\n",
      "True\n"
     ]
    }
   ],
   "source": [
    "print(tf.__version__)\n",
    "print(tf.test.is_gpu_available())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "cell_style": "split",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 52
    },
    "colab_type": "code",
    "id": "U4b9O6exjOtV",
    "outputId": "9755755c-e0fd-40a8-8261-d5b9fb9e6805"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.4.0\n",
      "True\n"
     ]
    }
   ],
   "source": [
    "print(torch.__version__)\n",
    "print(torch.cuda.is_available())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Bitdt9rbjOtj"
   },
   "source": [
    "### Data Loading\n",
    "MINST data set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "rvqodBI8jOtk"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "batch_size=200\n",
    "learning_rate=0.01\n",
    "epochs=10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "cell_style": "split",
    "colab": {},
    "colab_type": "code",
    "id": "zP9R3JXzjOtw"
   },
   "outputs": [],
   "source": [
    "(x, y),(x_test, y_test) = keras.datasets.mnist.load_data()\n",
    "\n",
    "ds_train = tf.data.Dataset.from_tensor_slices((x,y))\n",
    "ds_test = tf.data.Dataset.from_tensor_slices((x_test, y_test))\n",
    "\n",
    "def preprocess(x, y):\n",
    "  x = (tf.cast(x, tf.float32)/255)-0.1307\n",
    "  y = tf.cast(y, tf.int32)\n",
    "#   y = tf.one_hot(y,depth=10)   \n",
    "  return x, y\n",
    "\n",
    "ds_train = ds_train.map(preprocess).shuffle(1000).batch(batch_size)\n",
    "ds_test = ds_test.map(preprocess).shuffle(1000).batch(batch_size)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "cell_style": "split",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 322,
     "referenced_widgets": [
      "d1228d0da4fe431ab876e1767f93486b",
      "cd2b2f1bea8c44d1be2c8b1ed96e6b1d",
      "9ab389cdb3664dc9a5e5d73a131de3cf",
      "072302059898401c804ee7034832f9f9",
      "ab631f0323c44fd28bca2122e78972fd",
      "9dadb04364204ba580546c5ed46b533d",
      "45c56abd3b4340c6973365a8505a1489",
      "270a2223fc304ab1bbe6570299971020",
      "c2489720a216407c86df1dac212e6860",
      "6a5c3b4996f44a9a886ae2a749991633",
      "c293b4df711c4728b743a9d81f9a8d92",
      "31f3098f51774b12a29bd4457b9a627c",
      "09f1267e88274369a38f089ae15ece1e",
      "1720d7f79cf44c86957f956e3d001b92",
      "eead47fd96a24e16ab296a71fa19e50e",
      "15a7be22a6d2460a9378feefdc631f4d",
      "cd3348b28a3d4c52927785ef57be38b5",
      "96464aca53164b549ba46339054ee2e3",
      "936cbfbcb00242bfb9687aa41e5da83e",
      "91f55b603d684df2a5cc6105edc095a8",
      "5ac3d619429e4705b3602264f0efabaf",
      "703b6d9d7cc4475a8156891551b3cb3d",
      "55c8e2d421fe41f7bfd17873af455104",
      "c085442b5b9843edbb7846eb85f9dcd0",
      "54175c440eaf41618b6c220ebdac6953",
      "b951953cae084fbb903835217eadc1d7",
      "131cbda720564706ba6e8e65ab852dd0",
      "d369f8986df94b5a815bb143fe2bae1b",
      "3b38db19f8a74ff3932b1326cb7793a1",
      "a5bf223930554049acfbec1722530722",
      "6f36fb23c8bf4b4e885961de455303fa",
      "c0e637c4e34941a596aed9045e00d800"
     ]
    },
    "colab_type": "code",
    "id": "ZpTeZe_5jOt0",
    "outputId": "13d4a0f3-1021-4056-a0c6-a349cc244102",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train_loader = torch.utils.data.DataLoader(\n",
    "    datasets.MNIST('../data', train=True, download=True,\n",
    "                   transform=transforms.Compose([\n",
    "                       transforms.ToTensor(),\n",
    "                       transforms.Normalize((0.1307,), (0.3081,))\n",
    "                   ])),\n",
    "    batch_size=batch_size, shuffle=True)\n",
    "test_loader = torch.utils.data.DataLoader(\n",
    "    datasets.MNIST('../data', train=False, transform=transforms.Compose([\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize((0.1307,), (0.3081,))\n",
    "    ])),\n",
    "    batch_size=batch_size, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Fljk_143jOt6"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "cell_style": "split",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "id": "ZkzfJGgyjOuF",
    "outputId": "b666c75a-9a5d-4d5a-a488-44e2fc37981e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'tensorflow.python.data.ops.dataset_ops.BatchDataset'>\n",
      "(200, 28, 28) (200,)\n"
     ]
    }
   ],
   "source": [
    "print(type(ds_test))\n",
    "image, label = next(iter(ds_test))\n",
    "print(image.shape, label.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "cell_style": "split",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "id": "8nOXusKljOuK",
    "outputId": "637a5ea5-1b9a-498d-c99e-cef50d5b2d75",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'torch.utils.data.dataloader.DataLoader'>\n",
      "torch.Size([200, 1, 28, 28]) torch.Size([200])\n"
     ]
    }
   ],
   "source": [
    "print(type(train_loader))\n",
    "[image, label] = next(iter(train_loader))\n",
    "print(image.shape, label.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Z5gDAhrrjOub"
   },
   "source": [
    "### Regularization\n",
    "* Tensorflow: layers.Desnse set parameter `kernel_regularizer`\n",
    "* Pytorch: optimizer parameter `weight_dacy`\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "cell_style": "split",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "wR0A8v4UjOuc",
    "outputId": "abb2a3f7-eb55-43d3-bb80-fc9fc4e152d0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:0, step:0 loss:2.3325395584106445\n",
      "accuracy:  0.2624\n",
      "epoch:0, step:100 loss:0.2057160884141922\n",
      "accuracy:  0.9387\n",
      "epoch:0, step:200 loss:0.14280173182487488\n",
      "accuracy:  0.9503\n",
      "epoch:1, step:0 loss:0.20926254987716675\n",
      "accuracy:  0.9531\n",
      "epoch:1, step:100 loss:0.06743501871824265\n",
      "accuracy:  0.9657\n",
      "epoch:1, step:200 loss:0.2079378068447113\n",
      "accuracy:  0.9671\n",
      "epoch:2, step:0 loss:0.15533088147640228\n",
      "accuracy:  0.9603\n",
      "epoch:2, step:100 loss:0.11234745383262634\n",
      "accuracy:  0.9693\n",
      "epoch:2, step:200 loss:0.0930759608745575\n",
      "accuracy:  0.9593\n",
      "epoch:3, step:0 loss:0.13589558005332947\n",
      "accuracy:  0.964\n",
      "epoch:3, step:100 loss:0.055012188851833344\n",
      "accuracy:  0.9626\n",
      "epoch:3, step:200 loss:0.08367767184972763\n",
      "accuracy:  0.9651\n",
      "epoch:4, step:0 loss:0.0876694992184639\n",
      "accuracy:  0.9662\n",
      "epoch:4, step:100 loss:0.07405173033475876\n",
      "accuracy:  0.9623\n",
      "epoch:4, step:200 loss:0.2387072741985321\n",
      "accuracy:  0.9656\n",
      "epoch:5, step:0 loss:0.09264291822910309\n",
      "accuracy:  0.9694\n",
      "epoch:5, step:100 loss:0.025223366916179657\n",
      "accuracy:  0.9677\n",
      "epoch:5, step:200 loss:0.074891097843647\n",
      "accuracy:  0.9697\n",
      "epoch:6, step:0 loss:0.06347575038671494\n",
      "accuracy:  0.9662\n",
      "epoch:6, step:100 loss:0.03868771344423294\n",
      "accuracy:  0.9644\n",
      "epoch:6, step:200 loss:0.029421212151646614\n",
      "accuracy:  0.9701\n",
      "epoch:7, step:0 loss:0.058778028935194016\n",
      "accuracy:  0.967\n",
      "epoch:7, step:100 loss:0.06942695379257202\n",
      "accuracy:  0.9678\n",
      "epoch:7, step:200 loss:0.14350733160972595\n",
      "accuracy:  0.9621\n",
      "epoch:8, step:0 loss:0.048184070736169815\n",
      "accuracy:  0.9667\n",
      "epoch:8, step:100 loss:0.0065822238102555275\n",
      "accuracy:  0.9739\n",
      "epoch:8, step:200 loss:0.08629937469959259\n",
      "accuracy:  0.9693\n",
      "epoch:9, step:0 loss:0.16488642990589142\n",
      "accuracy:  0.9693\n",
      "epoch:9, step:100 loss:0.030548984184861183\n",
      "accuracy:  0.9653\n",
      "epoch:9, step:200 loss:0.026787899434566498\n",
      "accuracy:  0.9706\n"
     ]
    }
   ],
   "source": [
    "class FC_model(keras.Model):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "    \n",
    "        # Regulariztion applied here\n",
    "        self.model = keras.Sequential(\n",
    "            [layers.Dense(200, kernel_regularizer=keras.regularizers.l2(0.001)),\n",
    "            layers.ReLU(),\n",
    "            layers.Dense(100,kernel_regularizer=keras.regularizers.l2(0.001)),\n",
    "            layers.ReLU(),\n",
    "            layers.Dense(10)]\n",
    "            )\n",
    "    \n",
    "    def call(self,x):\n",
    "        x = self.model(x)\n",
    "        \n",
    "        return x\n",
    "    \n",
    "model = FC_model()\n",
    "optimizer = tf.optimizers.Adam(learning_rate)\n",
    "    \n",
    "for epoch in range(epochs):\n",
    "    \n",
    "    for step, (x, y) in enumerate(ds_train):\n",
    "        x = tf.reshape(x, [-1, 28*28])\n",
    "        with tf.GradientTape() as tape:            \n",
    "            logits = model(x)\n",
    "            \n",
    "            losses = tf.losses.sparse_categorical_crossentropy(y,logits,from_logits=True)\n",
    "            loss = tf.reduce_mean(losses)\n",
    "            \n",
    "        grads = tape.gradient(loss, model.variables)\n",
    "        \n",
    "        optimizer.apply_gradients(zip(grads, model.variables))\n",
    "        \n",
    "        if(step%100==0):\n",
    "            print(\"epoch:{}, step:{} loss:{}\".\n",
    "                  format(epoch, step, loss.numpy()))\n",
    "            \n",
    "            \n",
    "#             test accuracy: \n",
    "            total_correct = 0\n",
    "            total_num = 0\n",
    "            \n",
    "            for x_test, y_test in ds_test:\n",
    "                x_test = tf.reshape(x_test, [-1, 28*28])\n",
    "                y_pred = tf.argmax(model(x_test),axis=1)\n",
    "                y_pred = tf.cast(y_pred, tf.int32)\n",
    "                correct = tf.cast((y_pred == y_test), tf.int32)\n",
    "                correct = tf.reduce_sum(correct)\n",
    "                \n",
    "                total_correct += int(correct)\n",
    "                total_num += x_test.shape[0]\n",
    "        \n",
    "            \n",
    "            accuracy = total_correct/total_num\n",
    "            print('accuracy: ', accuracy)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "cell_style": "split",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "j0re9gEWjOup",
    "outputId": "a8746624-5d8d-430d-9827-dcec4f800651"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:0, step:0, loss:2.3211936950683594\n",
      "accuracy:  0.35179999470710754\n",
      "epoch:0, step:100, loss:0.3450058102607727\n",
      "accuracy:  0.9012999534606934\n",
      "epoch:0, step:200, loss:0.25054946541786194\n",
      "accuracy:  0.904699981212616\n",
      "epoch:1, step:0, loss:0.23051348328590393\n",
      "accuracy:  0.9236999750137329\n",
      "epoch:1, step:100, loss:0.18832090497016907\n",
      "accuracy:  0.9197999835014343\n",
      "epoch:1, step:200, loss:0.262503981590271\n",
      "accuracy:  0.9164999723434448\n",
      "epoch:2, step:0, loss:0.3107120394706726\n",
      "accuracy:  0.9044999480247498\n",
      "epoch:2, step:100, loss:0.2616995871067047\n",
      "accuracy:  0.9297999739646912\n",
      "epoch:2, step:200, loss:0.18142449855804443\n",
      "accuracy:  0.9273999929428101\n",
      "epoch:3, step:0, loss:0.28620439767837524\n",
      "accuracy:  0.9300999641418457\n",
      "epoch:3, step:100, loss:0.2452820986509323\n",
      "accuracy:  0.941100001335144\n",
      "epoch:3, step:200, loss:0.24611301720142365\n",
      "accuracy:  0.9420999884605408\n",
      "epoch:4, step:0, loss:0.2318108230829239\n",
      "accuracy:  0.9275999665260315\n",
      "epoch:4, step:100, loss:0.20092058181762695\n",
      "accuracy:  0.9327999949455261\n",
      "epoch:4, step:200, loss:0.1727539300918579\n",
      "accuracy:  0.9294999837875366\n",
      "epoch:5, step:0, loss:0.1918669193983078\n",
      "accuracy:  0.9411999583244324\n",
      "epoch:5, step:100, loss:0.2087152898311615\n",
      "accuracy:  0.9218999743461609\n",
      "epoch:5, step:200, loss:0.2244347333908081\n",
      "accuracy:  0.9362999796867371\n",
      "epoch:6, step:0, loss:0.2195575088262558\n",
      "accuracy:  0.9314999580383301\n",
      "epoch:6, step:100, loss:0.2261180281639099\n",
      "accuracy:  0.9321999549865723\n",
      "epoch:6, step:200, loss:0.2958630323410034\n",
      "accuracy:  0.9355999827384949\n",
      "epoch:7, step:0, loss:0.27482104301452637\n",
      "accuracy:  0.9282999634742737\n",
      "epoch:7, step:100, loss:0.18283149600028992\n",
      "accuracy:  0.932699978351593\n",
      "epoch:7, step:200, loss:0.2549021244049072\n",
      "accuracy:  0.9375\n",
      "epoch:8, step:0, loss:0.1737496256828308\n",
      "accuracy:  0.9304999709129333\n",
      "epoch:8, step:100, loss:0.11850783973932266\n",
      "accuracy:  0.9296999573707581\n",
      "epoch:8, step:200, loss:0.2949375510215759\n",
      "accuracy:  0.9416999816894531\n",
      "epoch:9, step:0, loss:0.22461959719657898\n",
      "accuracy:  0.9329999685287476\n",
      "epoch:9, step:100, loss:0.22229346632957458\n",
      "accuracy:  0.9355999827384949\n",
      "epoch:9, step:200, loss:0.21436114609241486\n",
      "accuracy:  0.9258999824523926\n"
     ]
    }
   ],
   "source": [
    "class FC_NN(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "    \n",
    "        self.model = nn.Sequential(\n",
    "            nn.Linear(28*28, 200),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Linear(200, 100),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Linear(100,10)\n",
    "            )\n",
    "    \n",
    "    def forward(self, x):\n",
    "        x = self.model(x)\n",
    "        \n",
    "        return x\n",
    "device = torch.device('cuda:0')\n",
    "\n",
    "network = FC_NN().to(device)\n",
    "\n",
    "# L2 regularization == weight_decay in the optimizers\n",
    "optimizer = torch.optim.Adam(network.parameters(),\n",
    "                            lr=learning_rate, weight_decay=0.01)\n",
    "criteon = torch.nn.CrossEntropyLoss().to(device)\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    \n",
    "    for step, (x, y) in enumerate(train_loader):\n",
    "        x = x.reshape(-1,28*28)\n",
    "        \n",
    "        x, y = x.to(device), y.to(device)\n",
    "        \n",
    "        logits = network(x)\n",
    "        loss = criteon(logits, y)\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        if(step%100 == 0):\n",
    "            print(\"epoch:{}, step:{}, loss:{}\".\n",
    "                  format(epoch, step, loss.item()))\n",
    "        \n",
    "#             test accuracy\n",
    "            total_correct = 0\n",
    "            total_num = 0    \n",
    "\n",
    "            for x_test, y_test in test_loader:\n",
    "                    x_test = x_test.reshape(-1,28*28)\n",
    "                    x_test, y_test = x_test.to(device), y_test.to(device)\n",
    "\n",
    "                    y_pred = network(x_test)\n",
    "                    y_pred = torch.argmax(y_pred, dim = 1)\n",
    "                    correct = y_pred == y_test\n",
    "                    correct = correct.sum()\n",
    "\n",
    "                    total_correct += correct\n",
    "                    total_num += x_test.shape[0]\n",
    "\n",
    "            acc = total_correct.float()/total_num\n",
    "            print(\"accuracy: \", acc.item())\n",
    "                \n",
    "                "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "A1xzOv11mWnU"
   },
   "source": [
    "### Dropout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "cell_style": "split",
    "colab": {},
    "colab_type": "code",
    "id": "TIP_isnKmWnX",
    "outputId": "babb84c7-a851-4ced-c76c-dcf8b79bc31f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:0, step:0 loss:2.366990566253662\n",
      "accuracy:  0.4342\n",
      "epoch:0, step:100 loss:0.1408572494983673\n",
      "accuracy:  0.9444\n",
      "epoch:0, step:200 loss:0.1306784301996231\n",
      "accuracy:  0.9476\n",
      "epoch:1, step:0 loss:0.18211042881011963\n",
      "accuracy:  0.9523\n",
      "epoch:1, step:100 loss:0.1134713813662529\n",
      "accuracy:  0.9645\n",
      "epoch:1, step:200 loss:0.10413701087236404\n",
      "accuracy:  0.9562\n",
      "epoch:2, step:0 loss:0.10256875306367874\n",
      "accuracy:  0.9589\n",
      "epoch:2, step:100 loss:0.08379142731428146\n",
      "accuracy:  0.9636\n",
      "epoch:2, step:200 loss:0.06404948979616165\n",
      "accuracy:  0.9685\n",
      "epoch:3, step:0 loss:0.031297821551561356\n",
      "accuracy:  0.9668\n",
      "epoch:3, step:100 loss:0.06204662472009659\n",
      "accuracy:  0.9681\n",
      "epoch:3, step:200 loss:0.039909422397613525\n",
      "accuracy:  0.9734\n",
      "epoch:4, step:0 loss:0.12970837950706482\n",
      "accuracy:  0.9707\n",
      "epoch:4, step:100 loss:0.1105945035815239\n",
      "accuracy:  0.9647\n",
      "epoch:4, step:200 loss:0.1333925724029541\n",
      "accuracy:  0.969\n",
      "epoch:5, step:0 loss:0.0438968688249588\n",
      "accuracy:  0.9689\n",
      "epoch:5, step:100 loss:0.06427070498466492\n",
      "accuracy:  0.9751\n",
      "epoch:5, step:200 loss:0.04650873690843582\n",
      "accuracy:  0.9689\n",
      "epoch:6, step:0 loss:0.06343336403369904\n",
      "accuracy:  0.9697\n",
      "epoch:6, step:100 loss:0.07626428455114365\n",
      "accuracy:  0.974\n",
      "epoch:6, step:200 loss:0.055490318685770035\n",
      "accuracy:  0.9682\n",
      "epoch:7, step:0 loss:0.1507387012243271\n",
      "accuracy:  0.9664\n",
      "epoch:7, step:100 loss:0.09286125004291534\n",
      "accuracy:  0.9731\n",
      "epoch:7, step:200 loss:0.07143430411815643\n",
      "accuracy:  0.9689\n",
      "epoch:8, step:0 loss:0.08946146070957184\n",
      "accuracy:  0.9714\n",
      "epoch:8, step:100 loss:0.027022486552596092\n",
      "accuracy:  0.9722\n",
      "epoch:8, step:200 loss:0.0307907834649086\n",
      "accuracy:  0.9683\n",
      "epoch:9, step:0 loss:0.07391297072172165\n",
      "accuracy:  0.9693\n",
      "epoch:9, step:100 loss:0.08514115959405899\n",
      "accuracy:  0.9728\n",
      "epoch:9, step:200 loss:0.05861741676926613\n",
      "accuracy:  0.9687\n"
     ]
    }
   ],
   "source": [
    "class FC_model(keras.Model):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "    \n",
    "        self.model = keras.Sequential(\n",
    "            [layers.Dense(200),\n",
    "            layers.ReLU(),\n",
    "            layers.Dropout(0.4),\n",
    "            layers.Dense(100),\n",
    "            layers.ReLU(),\n",
    "            layers.Dropout(0.4),             \n",
    "            layers.Dense(10)]\n",
    "            )\n",
    "    \n",
    "    def call(self,x):\n",
    "        x = self.model(x)\n",
    "        \n",
    "        return x\n",
    "    \n",
    "model = FC_model()\n",
    "optimizer = tf.optimizers.Adam(learning_rate)\n",
    "    \n",
    "for epoch in range(epochs):\n",
    "    \n",
    "    for step, (x, y) in enumerate(ds_train):\n",
    "        x = tf.reshape(x, [-1, 28*28])\n",
    "        with tf.GradientTape() as tape:            \n",
    "            logits = model(x)\n",
    "            \n",
    "            losses = tf.losses.sparse_categorical_crossentropy(y,logits,from_logits=True)\n",
    "            loss = tf.reduce_mean(losses)\n",
    "            \n",
    "        grads = tape.gradient(loss, model.variables)\n",
    "        \n",
    "        optimizer.apply_gradients(zip(grads, model.variables))\n",
    "        \n",
    "        if(step%100==0):\n",
    "            print(\"epoch:{}, step:{} loss:{}\".\n",
    "                  format(epoch, step, loss.numpy()))\n",
    "            \n",
    "            \n",
    "#             test accuracy: \n",
    "            total_correct = 0\n",
    "            total_num = 0\n",
    "            \n",
    "            for x_test, y_test in ds_test:\n",
    "                x_test = tf.reshape(x_test, [-1, 28*28])\n",
    "                y_pred = tf.argmax(model(x_test),axis=1)\n",
    "                y_pred = tf.cast(y_pred, tf.int32)\n",
    "                correct = tf.cast((y_pred == y_test), tf.int32)\n",
    "                correct = tf.reduce_sum(correct)\n",
    "                \n",
    "                total_correct += int(correct)\n",
    "                total_num += x_test.shape[0]\n",
    "        \n",
    "            \n",
    "            accuracy = total_correct/total_num\n",
    "            print('accuracy: ', accuracy)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "cell_style": "split",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "G9edsLH0mWnb",
    "outputId": "60d99f38-8dd1-4cde-cdec-e22e68493aec"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:0, step:0, loss:2.3078196048736572\n",
      "accuracy:  0.18709999322891235\n",
      "epoch:0, step:100, loss:0.41146090626716614\n",
      "accuracy:  0.8745999932289124\n",
      "epoch:0, step:200, loss:0.33413898944854736\n",
      "accuracy:  0.8973999619483948\n",
      "epoch:1, step:0, loss:0.4246741235256195\n",
      "accuracy:  0.8872999548912048\n",
      "epoch:1, step:100, loss:0.39421287178993225\n",
      "accuracy:  0.8986999988555908\n",
      "epoch:1, step:200, loss:0.20466510951519012\n",
      "accuracy:  0.8973999619483948\n",
      "epoch:2, step:0, loss:0.40461957454681396\n",
      "accuracy:  0.8955000042915344\n",
      "epoch:2, step:100, loss:0.5728944540023804\n",
      "accuracy:  0.9085999727249146\n",
      "epoch:2, step:200, loss:0.17806629836559296\n",
      "accuracy:  0.9031999707221985\n",
      "epoch:3, step:0, loss:0.5936587452888489\n",
      "accuracy:  0.9023000001907349\n",
      "epoch:3, step:100, loss:0.30897387862205505\n",
      "accuracy:  0.9096999764442444\n",
      "epoch:3, step:200, loss:0.6682959198951721\n",
      "accuracy:  0.9140999913215637\n",
      "epoch:4, step:0, loss:0.3526836335659027\n",
      "accuracy:  0.9041999578475952\n",
      "epoch:4, step:100, loss:0.4131539463996887\n",
      "accuracy:  0.9066999554634094\n",
      "epoch:4, step:200, loss:0.31797751784324646\n",
      "accuracy:  0.9083999991416931\n",
      "epoch:5, step:0, loss:0.5003743171691895\n",
      "accuracy:  0.909500002861023\n",
      "epoch:5, step:100, loss:0.401574969291687\n",
      "accuracy:  0.9138000011444092\n",
      "epoch:5, step:200, loss:0.3756680190563202\n",
      "accuracy:  0.911899983882904\n",
      "epoch:6, step:0, loss:0.3325634002685547\n",
      "accuracy:  0.9106000065803528\n",
      "epoch:6, step:100, loss:0.1378013640642166\n",
      "accuracy:  0.915399968624115\n",
      "epoch:6, step:200, loss:0.31622713804244995\n",
      "accuracy:  0.9061999917030334\n",
      "epoch:7, step:0, loss:0.36755988001823425\n",
      "accuracy:  0.9009999632835388\n",
      "epoch:7, step:100, loss:0.23866689205169678\n",
      "accuracy:  0.9169999957084656\n",
      "epoch:7, step:200, loss:0.4614146053791046\n",
      "accuracy:  0.9138000011444092\n",
      "epoch:8, step:0, loss:0.270405650138855\n",
      "accuracy:  0.9073999524116516\n",
      "epoch:8, step:100, loss:0.21292169392108917\n",
      "accuracy:  0.9178999662399292\n",
      "epoch:8, step:200, loss:0.3419126570224762\n",
      "accuracy:  0.9086999893188477\n",
      "epoch:9, step:0, loss:0.2932605743408203\n",
      "accuracy:  0.9114999771118164\n",
      "epoch:9, step:100, loss:0.2694905400276184\n",
      "accuracy:  0.9164999723434448\n",
      "epoch:9, step:200, loss:0.38846394419670105\n",
      "accuracy:  0.9126999974250793\n"
     ]
    }
   ],
   "source": [
    "class FC_NN(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "    \n",
    "        self.model = nn.Sequential(\n",
    "            nn.Linear(28*28, 200),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Dropout(0.4),\n",
    "            nn.Linear(200, 100),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Dropout(0.4),\n",
    "            nn.Linear(100,10)\n",
    "            )\n",
    "    \n",
    "    def forward(self, x):\n",
    "        x = self.model(x)\n",
    "        \n",
    "        return x\n",
    "device = torch.device('cuda:0')\n",
    "\n",
    "network = FC_NN().to(device)        \n",
    "optimizer = torch.optim.Adam(network.parameters(),\n",
    "                            lr=learning_rate)\n",
    "criteon = torch.nn.CrossEntropyLoss().to(device)\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    \n",
    "    for step, (x, y) in enumerate(train_loader):\n",
    "        x = x.reshape(-1,28*28)\n",
    "        \n",
    "        x, y = x.to(device), y.to(device)\n",
    "        \n",
    "        logits = network(x)\n",
    "        loss = criteon(logits, y)\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        if(step%100 == 0):\n",
    "            print(\"epoch:{}, step:{}, loss:{}\".\n",
    "                  format(epoch, step, loss.item()))\n",
    "        \n",
    "#             test accuracy\n",
    "            total_correct = 0\n",
    "            total_num = 0    \n",
    "\n",
    "            for x_test, y_test in test_loader:\n",
    "                    x_test = x_test.reshape(-1,28*28)\n",
    "                    x_test, y_test = x_test.to(device), y_test.to(device)\n",
    "\n",
    "                    y_pred = network(x_test)\n",
    "                    y_pred = torch.argmax(y_pred, dim = 1)\n",
    "                    correct = y_pred == y_test\n",
    "                    correct = correct.sum()\n",
    "\n",
    "                    total_correct += correct\n",
    "                    total_num += x_test.shape[0]\n",
    "\n",
    "            acc = total_correct.float()/total_num\n",
    "            print(\"accuracy: \", acc.item())\n",
    "                \n",
    "                "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "WHOMA3_RrMNQ"
   },
   "source": [
    "### Learing Rate Decay"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "cell_style": "split",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "Z-Tu09SurMNT",
    "outputId": "817d2466-fcc9-4352-8f7c-20ed12f626af"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:0, step:0 loss:2.288949966430664\n",
      "accuracy:  0.3045 learing rate:  0.199\n",
      "epoch:0, step:100 loss:0.2368682622909546\n",
      "accuracy:  0.902 learing rate:  0.12054837\n",
      "epoch:0, step:200 loss:0.22905904054641724\n",
      "accuracy:  0.9212 learing rate:  0.073024675\n",
      "epoch:1, step:0 loss:0.26692017912864685\n",
      "accuracy:  0.9255 learing rate:  0.044236213\n",
      "epoch:1, step:100 loss:0.29251259565353394\n",
      "accuracy:  0.9285 learing rate:  0.02679701\n",
      "epoch:1, step:200 loss:0.27387234568595886\n",
      "accuracy:  0.9313 learing rate:  0.01623284\n",
      "epoch:2, step:0 loss:0.26345327496528625\n",
      "accuracy:  0.9321 learing rate:  0.009833382\n",
      "epoch:2, step:100 loss:0.28329917788505554\n",
      "accuracy:  0.9322 learing rate:  0.005956773\n",
      "epoch:2, step:200 loss:0.22056129574775696\n",
      "accuracy:  0.9325 learing rate:  0.0036084396\n",
      "epoch:3, step:0 loss:0.2406102567911148\n",
      "accuracy:  0.9332 learing rate:  0.0021858872\n",
      "epoch:3, step:100 loss:0.23601095378398895\n",
      "accuracy:  0.9331 learing rate:  0.0013241466\n",
      "epoch:3, step:200 loss:0.19479002058506012\n",
      "accuracy:  0.9331 learing rate:  0.00080212957\n",
      "epoch:4, step:0 loss:0.3106921315193176\n",
      "accuracy:  0.933 learing rate:  0.00048590655\n",
      "epoch:4, step:100 loss:0.19223567843437195\n",
      "accuracy:  0.9331 learing rate:  0.00029434788\n",
      "epoch:4, step:200 loss:0.1722545027732849\n",
      "accuracy:  0.9331 learing rate:  0.00017830727\n",
      "epoch:5, step:0 loss:0.283220112323761\n",
      "accuracy:  0.933 learing rate:  0.00010801336\n",
      "epoch:5, step:100 loss:0.290651798248291\n",
      "accuracy:  0.933 learing rate:  6.5431326e-05\n",
      "epoch:5, step:200 loss:0.2376844733953476\n",
      "accuracy:  0.9331 learing rate:  3.9636365e-05\n",
      "epoch:6, step:0 loss:0.3100353479385376\n",
      "accuracy:  0.933 learing rate:  2.4010551e-05\n",
      "epoch:6, step:100 loss:0.293476939201355\n",
      "accuracy:  0.9331 learing rate:  1.4544881e-05\n",
      "epoch:6, step:200 loss:0.2881743907928467\n",
      "accuracy:  0.9331 learing rate:  8.810864e-06\n",
      "epoch:7, step:0 loss:0.2384854257106781\n",
      "accuracy:  0.9331 learing rate:  5.337363e-06\n",
      "epoch:7, step:100 loss:0.21170015633106232\n",
      "accuracy:  0.9331 learing rate:  3.2332189e-06\n",
      "epoch:7, step:200 loss:0.15076282620429993\n",
      "accuracy:  0.9331 learing rate:  1.9585896e-06\n",
      "epoch:8, step:0 loss:0.20046070218086243\n",
      "accuracy:  0.9331 learing rate:  1.1864564e-06\n",
      "epoch:8, step:100 loss:0.3313523828983307\n",
      "accuracy:  0.9331 learing rate:  9.955421e-07\n",
      "epoch:8, step:200 loss:0.22597776353359222\n",
      "accuracy:  0.9331 learing rate:  9.955421e-07\n",
      "epoch:9, step:0 loss:0.25489330291748047\n",
      "accuracy:  0.9331 learing rate:  9.955421e-07\n",
      "epoch:9, step:100 loss:0.2508421242237091\n",
      "accuracy:  0.9331 learing rate:  9.955421e-07\n",
      "epoch:9, step:200 loss:0.13665063679218292\n",
      "accuracy:  0.9331 learing rate:  9.955421e-07\n"
     ]
    }
   ],
   "source": [
    "class FC_model(keras.Model):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "    \n",
    "        self.model = keras.Sequential(\n",
    "            [layers.Dense(200),\n",
    "            layers.ReLU(),\n",
    "            layers.Dense(100),\n",
    "            layers.ReLU(),             \n",
    "            layers.Dense(10)]\n",
    "            )\n",
    "    \n",
    "    def call(self,x):\n",
    "        x = self.model(x)\n",
    "        \n",
    "        return x\n",
    "    \n",
    "model = FC_model()\n",
    "\n",
    "# set initial learning rate and minimum learning rate\n",
    "lr_init = 0.2\n",
    "lr_min = 1e-6\n",
    "lr_decay = 0.995\n",
    "\n",
    "optimizer = tf.optimizers.SGD(learning_rate=lr_init)\n",
    "\n",
    "global_step = 0\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    \n",
    "    for step, (x, y) in enumerate(ds_train):\n",
    "        x = tf.reshape(x, [-1, 28*28])\n",
    "        with tf.GradientTape() as tape:            \n",
    "            logits = model(x)\n",
    "            \n",
    "            losses = tf.losses.sparse_categorical_crossentropy(y,logits,from_logits=True)\n",
    "            loss = tf.reduce_mean(losses)\n",
    "            \n",
    "        grads = tape.gradient(loss, model.variables)\n",
    "        \n",
    "        # Decay learning rate here \n",
    "        if optimizer.learning_rate > lr_min:\n",
    "          optimizer.learning_rate = optimizer.learning_rate * lr_decay\n",
    "        \n",
    "        optimizer.apply_gradients(zip(grads, model.variables))\n",
    "        \n",
    "        if(step%100==0):\n",
    "            print(\"epoch:{}, step:{} loss:{}\".\n",
    "                  format(epoch, step, loss.numpy()))\n",
    "            \n",
    "            \n",
    "#             test accuracy: \n",
    "            total_correct = 0\n",
    "            total_num = 0\n",
    "            \n",
    "            for x_test, y_test in ds_test:\n",
    "                x_test = tf.reshape(x_test, [-1, 28*28])\n",
    "                y_pred = tf.argmax(model(x_test),axis=1)\n",
    "                y_pred = tf.cast(y_pred, tf.int32)\n",
    "                correct = tf.cast((y_pred == y_test), tf.int32)\n",
    "                correct = tf.reduce_sum(correct)\n",
    "                \n",
    "                total_correct += int(correct)\n",
    "                total_num += x_test.shape[0]\n",
    "        \n",
    "            \n",
    "            accuracy = total_correct/total_num\n",
    "            print('accuracy: ', accuracy, 'learing rate: ', optimizer.learning_rate.numpy())\n",
    "\n",
    "        global_step += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "cell_style": "split",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "_v-MtJfErMNY",
    "outputId": "22994b58-5101-4325-97a2-e195675d7365"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:0, step:0, loss:2.323620557785034\n",
      "accuracy:  0.08709999918937683 learning rate:  0.01\n",
      "epoch:0, step:100, loss:1.9669822454452515\n",
      "accuracy:  0.66839998960495 learning rate:  0.01\n",
      "epoch:0, step:200, loss:1.1098418235778809\n",
      "accuracy:  0.7950999736785889 learning rate:  0.01\n",
      "epoch:1, step:0, loss:0.6557897329330444\n",
      "accuracy:  0.8479999899864197 learning rate:  0.01\n",
      "epoch:1, step:100, loss:0.5625755786895752\n",
      "accuracy:  0.8687999844551086 learning rate:  0.01\n",
      "epoch:1, step:200, loss:0.5705286860466003\n",
      "accuracy:  0.8807999491691589 learning rate:  0.01\n",
      "epoch:2, step:0, loss:0.4688332676887512\n",
      "accuracy:  0.8881999850273132 learning rate:  0.01\n",
      "epoch:2, step:100, loss:0.3717648684978485\n",
      "accuracy:  0.894599974155426 learning rate:  0.01\n",
      "epoch:2, step:200, loss:0.36960285902023315\n",
      "accuracy:  0.8983999490737915 learning rate:  0.01\n",
      "epoch:3, step:0, loss:0.37219443917274475\n",
      "accuracy:  0.9024999737739563 learning rate:  0.001\n",
      "epoch:3, step:100, loss:0.4337742328643799\n",
      "accuracy:  0.9032999873161316 learning rate:  0.001\n",
      "epoch:3, step:200, loss:0.2757149040699005\n",
      "accuracy:  0.903499960899353 learning rate:  0.0001\n",
      "epoch:4, step:0, loss:0.393237829208374\n",
      "accuracy:  0.9032999873161316 learning rate:  0.0001\n",
      "epoch:4, step:100, loss:0.3285243511199951\n",
      "accuracy:  0.9034000039100647 learning rate:  0.0001\n",
      "epoch:4, step:200, loss:0.29251325130462646\n",
      "accuracy:  0.9032999873161316 learning rate:  1e-05\n",
      "epoch:5, step:0, loss:0.36429882049560547\n",
      "accuracy:  0.9032999873161316 learning rate:  1e-05\n",
      "epoch:5, step:100, loss:0.4052523076534271\n",
      "accuracy:  0.9032999873161316 learning rate:  1e-05\n",
      "epoch:5, step:200, loss:0.3859458863735199\n",
      "accuracy:  0.9032999873161316 learning rate:  1.0000000000000002e-06\n",
      "epoch:6, step:0, loss:0.42303356528282166\n",
      "accuracy:  0.9032999873161316 learning rate:  1.0000000000000002e-06\n",
      "epoch:6, step:100, loss:0.4477410912513733\n",
      "accuracy:  0.9032999873161316 learning rate:  1.0000000000000002e-07\n",
      "epoch:6, step:200, loss:0.29990869760513306\n",
      "accuracy:  0.9032999873161316 learning rate:  1.0000000000000002e-07\n",
      "epoch:7, step:0, loss:0.3107373118400574\n",
      "accuracy:  0.9032999873161316 learning rate:  1.0000000000000004e-08\n",
      "epoch:7, step:100, loss:0.3927546739578247\n",
      "accuracy:  0.9032999873161316 learning rate:  1.0000000000000004e-08\n",
      "epoch:7, step:200, loss:0.34210941195487976\n",
      "accuracy:  0.9032999873161316 learning rate:  1.0000000000000004e-08\n",
      "epoch:8, step:0, loss:0.3747851252555847\n",
      "accuracy:  0.9032999873161316 learning rate:  1.0000000000000004e-08\n",
      "epoch:8, step:100, loss:0.28887903690338135\n",
      "accuracy:  0.9032999873161316 learning rate:  1.0000000000000004e-08\n",
      "epoch:8, step:200, loss:0.35422202944755554\n",
      "accuracy:  0.9032999873161316 learning rate:  1.0000000000000004e-08\n",
      "epoch:9, step:0, loss:0.2731885313987732\n",
      "accuracy:  0.9032999873161316 learning rate:  1.0000000000000004e-08\n",
      "epoch:9, step:100, loss:0.40830329060554504\n",
      "accuracy:  0.9032999873161316 learning rate:  1.0000000000000004e-08\n",
      "epoch:9, step:200, loss:0.3329598903656006\n",
      "accuracy:  0.9032999873161316 learning rate:  1.0000000000000004e-08\n"
     ]
    }
   ],
   "source": [
    "class FC_NN(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "    \n",
    "        self.model = nn.Sequential(\n",
    "            nn.Linear(28*28, 200),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Linear(200, 100),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Linear(100,10)\n",
    "            )\n",
    "    \n",
    "    def forward(self, x):\n",
    "        x = self.model(x)\n",
    "        \n",
    "        return x\n",
    "device = torch.device('cuda:0')\n",
    "\n",
    "network = FC_NN().to(device)        \n",
    "optimizer = torch.optim.SGD(network.parameters(),\n",
    "                            lr=learning_rate)\n",
    "\n",
    "# learing rate will drop with implovement with number (patience) of epsodes\n",
    "scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, patience=200)\n",
    "\n",
    "criteon = torch.nn.CrossEntropyLoss().to(device)\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    \n",
    "    for step, (x, y) in enumerate(train_loader):\n",
    "        x = x.reshape(-1,28*28)\n",
    "        \n",
    "        x, y = x.to(device), y.to(device)\n",
    "        \n",
    "        logits = network(x)\n",
    "        loss = criteon(logits, y)\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        # Update learning rate here\n",
    "        scheduler.step(loss)\n",
    "        \n",
    "        if(step%100 == 0):\n",
    "            print(\"epoch:{}, step:{}, loss:{}\".\n",
    "                  format(epoch, step, loss.item()))\n",
    "        \n",
    "#             test accuracy\n",
    "            total_correct = 0\n",
    "            total_num = 0    \n",
    "\n",
    "            for x_test, y_test in test_loader:\n",
    "                    x_test = x_test.reshape(-1,28*28)\n",
    "                    x_test, y_test = x_test.to(device), y_test.to(device)\n",
    "\n",
    "                    y_pred = network(x_test)\n",
    "                    y_pred = torch.argmax(y_pred, dim = 1)\n",
    "                    correct = y_pred == y_test\n",
    "                    correct = correct.sum()\n",
    "\n",
    "                    total_correct += correct\n",
    "                    total_num += x_test.shape[0]\n",
    "\n",
    "            acc = total_correct.float()/total_num\n",
    "            print(\"accuracy: \", acc.item(), \"learning rate: \", optimizer.param_groups[0]['lr'])\n",
    "                \n",
    "                "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "S97SqxY0rwWn"
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "QsZ7_I90XrQu"
   },
   "source": [
    "Plot learning rate decay"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "eY2Ixa5MBrJo"
   },
   "outputs": [],
   "source": [
    "# Maximun global_step about 3000\n",
    "\n",
    "lr = 0.2\n",
    "min_lr = 1e-6\n",
    "\n",
    "global_step = 0\n",
    "decay_rate = 0.995\n",
    "\n",
    "recoder = {\"lr\":[],\"global_step\":[]}\n",
    "\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    \n",
    "    for step, (x, y) in enumerate(ds_train):\n",
    "      recoder['lr'].append(lr) \n",
    "      recoder['global_step'].append(global_step)\n",
    "      \n",
    "      # if lr > min_lr:\n",
    "      lr = lr * decay_rate\n",
    "\n",
    "      global_step += 1\n",
    "\n",
    "      "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 265
    },
    "colab_type": "code",
    "id": "5rySy8rXVhFH",
    "outputId": "ec38e232-bfe3-4bf0-f9ce-a83d43ccc32c"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAD4CAYAAADvsV2wAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dd3xUVf7/8dcnlR4pAWmhFwMBpSsQ\n1JUqggoq9lUUERGE3VX86u7irq5lV4oKIipiBVFQikhbNaFXpYQamgQpoUhvgfP7g3F/2SwlkGTu\nzOT9fDzmwdyTycz7eOOHw7mXD+acQ0REQl+Y1wFERMQ/VPBFRPIJFXwRkXxCBV9EJJ9QwRcRySci\nvA5wIaVKlXKVK1f2OoaISFBZunTpHudcbNbxgCz4ZnYLcEv16tVZsmSJ13FERIKKmW0913hAbuk4\n5yY753rExMR4HUVEJGQEZMEXEZHcp4IvIpJPBGTBN7NbzGzkgQMHvI4iIhIyArLgaw9fRCT3BWTB\nFxGR3KeCLyKST4RkwU9an87HC7Zy5oxaP4uI/CYgC35OL9pOXv4Lf/56Fd3eXcDmPUdyOZ2ISHAK\nyIKf04u2/+xaj9e61mPtjoO0G5LMiKSNZJw+k8spRUSCS0AW/JwyM+5sVJFZ/Vtxfa1YXvl2LbcN\nn8fqXw56HU1ExDMhWfB/U7pYAUbc15Dh9zZgx4FjdHprDq/PWMeJjNNeRxMR8buQLvhwdrXfIaEs\nM/u1otPV5Xjzu1RufmMOS7fu9zqaiIhfhXzB/03xwlEMuvNqRj/UmGMnT9N1xDxemJzCkRMZXkcT\nEfELvxV8M6tqZu+b2Zf++sxzub5Waab3S+T+ZpX4YO4W2g5JZvaGdC8jiYj4RbYKvpmNMrPdZrYq\ny3g7M1tnZqlmNuBC7+Gc2+Sc656TsLmlSHQEf+tcl3GPXUtUeBj3v7+Ip79czoGjp7yOJiKSZ7K7\nwh8NtMs8YGbhwDCgPRAP3G1m8WaWYGZTsjxK52rqXNKkSgmm9m1Jr+urMX7Zdm4anMS0VTu9jiUi\nkieyVfCdc8nAvizDTYBU38r9JDAW6OycW+mc65jlsTu7gcysh5ktMbMl6el5v9VSIDKcp9vVZuIT\nzYktEk3PT5bS69Ol7D50PM8/W0TEn3Kyh18e2JbpOM03dk5mVtLMRgDXmNmz53udc26kc66Rc65R\nbOz//JOMeaZu+Rgm9m7On9rWYtaa3bQelMz4pWk4p/YMIhIa/HbR1jm31znX0zlXzTn38oVe61U/\n/MjwMJ64oTpT+7Skeuki/OGL5Tz4wWLS9h/1aw4RkbyQk4K/HaiY6biCbyzoVS9dhC8eu5YXOtVh\nyZZ9tB2czEfzt6gZm4gEtZwU/MVADTOrYmZRQDdgUm6ECoR/ACUszHjwusrM6JdIw8ol+MvEFO4a\nOZ+N6Yc9yyQikhPZvS1zDDAfqGVmaWbW3TmXAfQGpgNrgHHOuZTcCBVI/8RhheKF+PChxvzrjvqs\n33WY9kNnM+z7VE6pGZuIBBkL5IuSjRo1ckuWLPE6xn/sPnScgZNSmLpyJ3XKFePVLvWoW17/DKOI\nBBYzW+qca5R1PCBbKwTSCj+z0kULMPzehoy4rwG7Dp6g87C5vDZtLcdPqRmbiAQ+rfAv04Gjp3jx\nm9V8sTSNqrGFea1LPRpVLuF1LBERrfBzW0yhSP55R30+ergJJzPOcMc78/nrxFUcVjM2EQlQWuHn\ngiMnMvjn9HV8OH8L5WIK8o/bE2hV039/aUxEJLOgWuEHm8LREQzsVIcve15LgcgwHhy1iP7jfuLX\noye9jiYi8h8BWfCDYUvnXBpWKsE3fVrS+4bqTPrpF24alMTUlTu8jiUiAmhLJ8+k/HKAZ8avYNX2\ng7SrcyV/61yH0sUKeB1LRPIBben4WZ1yMXzdqznPtKvNd+t2c9OgJMYt2aZmbCLiGRX8PBQRHsbj\n11djWt+W1L6yGE9/uYIHRi1i2z41YxMR/wvIgh+se/jnUzW2CGN7NOPvt9Zl2db9tBmczAdzN3Na\nzdhExI+0h+9n2389xnNfreSHdek0iLuC17rWo3rpol7HEpEQoj38AFH+ioJ88PvGDL6rPpv2HKHD\n0Dm89d0GNWMTkTyngu8BM+O2ayowq38rWtcpw79mrOeWN+ewMi00trBEJDCp4HuoVJFoht3TgHfu\nb8i+Iye5dfhcXvlWzdhEJG+o4AeAtnWuZGb/VtzRsAIjkjbSfuhsFm7a63UsEQkxAVnwQ+0uneyI\nKRjJK13q8ekjTck4c4a7Ri7g+a9Xcuj4Ka+jiUiI0F06AejoyQxen7GeUXM3U7ZYAV66LYEbapf2\nOpaIBAndpRNECkVF8OeO8Yx//DoKR0fw0OjF9Pv8J/YdUTM2Ebl8KvgBrEFccab0aUGf39Vg8vJf\naD0oiSkrflF7BhG5LCr4AS46Ipz+rWsy+ckWlC9ekN6f/UiPj5ey6+Bxr6OJSJDxa8E3s1vN7F0z\n+9zM2vjzs4PdVWWLMeHx63iuw1Ukr0/npkFJjF30s1b7IpJt2S74ZjbKzHab2aos4+3MbJ2ZpZrZ\ngAu9h3Pua+fco0BP4K7Li5x/RYSH8WhiVaY/lUh82WIMmLCSe99byM971YxNRC7uUlb4o4F2mQfM\nLBwYBrQH4oG7zSzezBLMbEqWR+bbTJ73fZ9chsqlCjPm0Wb847YEVqQdoM2QJN6bvUnN2ETkgiKy\n+0LnXLKZVc4y3ARIdc5tAjCzsUBn59zLQMes72FmBrwCfOucW3a5oQXCwox7msZxQ+1YnvtqFS9+\ns4YpK3bwWtd61CyjZmwi8r9yuodfHtiW6TjNN3Y+TwI3AV3NrOe5XmBmPcxsiZktSU9Pz2G80Fc2\npiDvP9iIod2u5ud9R7n5jdkMnbWBkxlqxiYi/y3bK/zc4Jx7A3jjIq8ZaWY7gFuioqIa+idZcDMz\nOl9dnhbVS/G3KasZPGs9U1eeXe3Xr3iF1/FEJEDkdIW/HaiY6biCbyxHnHOTnXM9YmJicvpW+UrJ\nItEM7XYN7z3QiAPHTnHb8Lm89M1qjp1UMzYRyXnBXwzUMLMqZhYFdAMm5TRUfuylk5tuii/DjP6J\ndGsSx7uzN9NuaDLzN6oZm0h+dym3ZY4B5gO1zCzNzLo75zKA3sB0YA0wzjmXkjdR5VIUKxDJP25L\n4LNHmwJw97sLeHbCSg6qGZtIvqXmafnAsZOnGTxrPe/N3kTpogV46ba6/O6qMl7HEpE8ElTN07Sl\nk7sKRoXzfx2uYkKv5sQUjKT7h0voM+ZH9h4+4XU0EfEjrfDzmZMZZ3j7h4289f0GikRHMLBTHTrV\nL8fZvyIhIqEgqFb4kneiIsLoe1MNvunTkkolC9N37E888uESdhw45nU0EcljAVnwtaWT92qWKcr4\nx6/j+ZuvYu7GPbQelMynC7dyRu0ZREKWtnSEn/ceZcCEFczbuJdmVUvwyu31qFyqsNexROQyaUtH\nziuuZCE+faQpr9yeQMr2g7QdkszI5I1knFZ7BpFQEpAFX1s6/mdmdGsSx8z+rWhZI5Z/TF1Ll7fn\nsXbnQa+jiUgu0ZaO/A/nHN+s3MFfJ6Zw4Ngpet1QnSduqEZ0RLjX0UQkG7SlI9lmZnSsV45Z/Vtx\nS/1yvPHvDXR8Yw7Lft7vdTQRyYGALPja0gkMxQtHMfiuq/ng9405fCKDLm/P4+9TVnP0ZIbX0UTk\nMmhLR7Ll0PFTvDptLZ8s+JmKJQryyu31aF69lNexROQctKUjOVK0QCQv3prA5z2aEREWxr3vLWTA\n+BUcOKZmbCLBQgVfLknTqiX5tm9LeraqxhdL02g9KIkZKTu9jiUi2aCCL5esQGQ4A9rX5utezSlZ\nJJoeHy/lic+WkX5IzdhEApkKvly2hAoxTOrdnD+2qcnMlF20HpzEVz+mEcjXhUTys4As+LpLJ3hE\nhofR+8YaTO3bgqqlCtPv8+U8NHox239VMzaRQKO7dCTXnD7j+Gj+Fl6bto4wgwEdruLeJnGEhan1\nsog/6S4dyXPhYcZDzaswo18i18QV589fr6LbyAVsSj/sdTQRQQVf8kDFEoX4uHsTXutaj7U7D9Ju\n6Gze/kHN2ES8poIvecLMuLNRRWb1b8UNtWJ5ddpabh0+l9W/qBmbiFdU8CVPlS5WgHfub8Tb9zZg\n54ETdHprDv+avo7jp057HU0k3/FbwTezq8xshJl9aWaP++tzJTC0TyjLrP6JdL66PG99n8rNb8xm\n6dZ9XscSyVeyVfDNbJSZ7TazVVnG25nZOjNLNbMBF3oP59wa51xP4E6g+eVHlmB1RaEoXr+zPh8+\n3ITjp87QdcR8Bk5K4cgJNWMT8YfsrvBHA+0yD5hZODAMaA/EA3ebWbyZJZjZlCyP0r7v6QR8A0zN\ntRlI0GlVM5bp/RJ5oFklRs/bQpvBySSvT/c6lkjIy/Z9+GZWGZjinKvrO74WGOica+s7fhbAOfdy\nNt7rG+fczef5Wg+gB0BcXFzDrVu3ZiufBKfFW/bxzPgVbEo/QteGFfjzzfHEFIr0OpZIUMuL+/DL\nA9syHaf5xs4X4Hoze8PM3uECK3zn3EjgBWBZVFRUDuJJMGhcuQRT+7Sk1/XV+OrH7dw0OIlpq3Z4\nHUskJPntoq1z7gfnXB/n3GPOuWEXee1k51yPmJgYf8UTDxWIDOfpdrWZ+ERzYotE0/OTZTz+yVJ2\nHzrudTSRkJKTgr8dqJjpuIJvLMfUSyd/qls+hom9m/OntrX499rdtB6UzJdL1YxNJLfkpOAvBmqY\nWRUziwK6AZNyI5RW+PlXZHgYT9xQnal9WlKjdBH++MVyHhi1iG37jnodTSToZfe2zDHAfKCWmaWZ\nWXfnXAbQG5gOrAHGOedSciOUVvhSvXQRxj12LX/rXIdlW/fTdkgyo+du5swZrfZFLpe6ZUrAS9t/\nlP/7ahXJ69NpVKk4r3SpR/XSRbyOJRKwgqpbplb4klmF4oX48KHGvH5HfTbsPkyHobMZ9n0qp9SM\nTeSSaIUvQSX90An+OmkVU1fuJL5sMV7rWo+65XWtRyQzrfAlJMQWjWb4vQ0ZcV8D0g+foPOwubw6\nba2asYlkg1b4ErQOHD3Fi9+s5oulaVQtVZhXu9ajceUSXscS8VxQrfBFsiOmUCT/vKM+H3dvwsnT\nZ7hjxHz+MnEVh9WMTeScArLga0tHLkXLGrFMfyqRh5pX5uMFW2k7OJkf1u32OpZIwNGWjoSUpVv3\n88z4FaTuPsztDcrz55vjKV5YPZkkf9GWjuQLDSsV55s+LXjyxupM+ukXWg9OYurKHWrPIIIKvoSg\n6Ihw/tCmFpN6t6BsTEF6fbqMnp8sZfdBNWOT/C0gC7728CU3xJcrxle9rmNA+9r8sC6d3w1KYtzi\nbVrtS76lPXzJFzalH2bAhJUs2ryPFtVL8fLtCVQsUcjrWCJ5Qnv4kq9VjS3C2Eeb8eKtdflp26+0\nGZzMqDmbOa1mbJKPqOBLvhEWZtzXrBIz+iXStGoJ/jZlNXeMmMeGXYe8jibiFyr4ku+Uu6IgH/y+\nMUPuuprNe45w8xtzePPfG9SMTUJeQBZ8XbSVvGZm3HpNeWb2b0WbOmV4feZ6bnlzDivSfvU6mkie\n0UVbEWBGyk6e/3oVew6f4NGWVenXuiYFIsO9jiVyWXTRVuQC2tS5kpn9W3FX44q8k7yJdkOSWbBp\nr9exRHKVCr6IT0zBSF6+vR6fPdKUMw66jVzAc1+t5NDxU15HE8kVKvgiWVxXvRTTnmrJIy2qMGbR\nz7QZnMz3a9WMTYKfCr7IORSKiuD5jvGMf/w6ikRH8NDoxTw19kf2HTnpdTSRy+bXgm9mhc1siZl1\n9Ofnilyua+KKM6VPC/r+rgZTVuyg9aAkJi//Re0ZJChlq+Cb2Sgz221mq7KMtzOzdWaWamYDsvFW\nzwDjLieoiFeiI8Lp17omU/q0oELxgjw55kce/WgpOw+oGZsEl+yu8EcD7TIPmFk4MAxoD8QDd5tZ\nvJklmNmULI/SZtYaWA1oM1SCUu0rizGhV3Oe63AVc1LTaT0oiTGLftZqX4JGtu/DN7PKwBTnXF3f\n8bXAQOdcW9/xswDOuZfP8/0vAYU5+5vDMeA259z//NVGM+sB9ACIi4truHXr1kubkYgfbNlzhAET\nVrBg0z6urVqSV7okUKlkYa9jiQB5cx9+eWBbpuM039g5Oeeec849BXwGvHuuYu973UjnXCPnXKPY\n2NgcxBPJO5VLFeazR5rxj9sSWLX9AG2HJPPe7E1qxiYBze936TjnRjvnplzoNWqtIMEgLMy4p2kc\nM/on0rxaKV78Zg23vz2PdTvVjE0CU04K/nagYqbjCr4xkXylbExB3nuwEUO7Xc22fUfp+OZshsxa\nz8kMNWOTwJKTgr8YqGFmVcwsCugGTMqNUM65yc65HjExMbnxdiJ5zszofHV5ZvZLpENCWYbM2sAt\nb87hp21qxiaBI7u3ZY4B5gO1zCzNzLo75zKA3sB0YA0wzjmXkhuhtKUjwapkkWiGdruG9x9sxIFj\np7h9+Fxe+mY1x06e9jqaiLpliuSVg8dP8cq3a/ls4c/ElSjEK10SuK5aKa9jST4QVN0ytcKXUFCs\nQCT/uC2BMY82wwzueXchz05YyUE1YxOPaIUv4gfHTp5m8Kz1vDd7E7FFo3np1gRuii/jdSwJUVrh\ni3ioYFQ4/9fhKr7q1ZzihaJ45KMlPDnmR/YePuF1NMlHtMIX8bOTGWcYkbSRN7/bQJHoCAZ2qkOn\n+uUwM6+jSYgIqhW+SCiLigijz+9q8E2fllQqWZi+Y3+i+4dL+OXXY15HkxAXkAVfWzqSH9QsU5Tx\nj1/HnzvGM3/jXtoMTubThVs5o/YMkke0pSMSAH7ee5Rnv1rB3NS9NK1Sgle61KNKKTVjk8ujLR2R\nABZXshCfdG/Kq10SWL3jIO2GJPNO0kYyTqs9g+QeFXyRAGFm3NU4jln9W5FYM5aXv13L7W/PY82O\ng15HkxARkAVfe/iSn5UpVoCR9zdk2D0N+OXXY9zy5hwGzVjHiQy1Z5Cc0R6+SADbf+Qkf5+ymgk/\nbqdG6SK82rUeDeKKex1LApz28EWCUPHCUQy662o+eKgxR05k0OXtefxt8mqOnszwOpoEIRV8kSBw\nQ63STO+XyH1NKzFq7mbaDklmzoY9XseSIKOCLxIkihaI5O+31uXzHs2ICAvjvvcX8vSXyzlwTM3Y\nJHsCsuDroq3I+TWtWpJv+7akZ6tqjF+2ndaDkpiestPrWBIEdNFWJIitTDvA0+NXsGbHQW5OKMvA\nTnWILRrtdSzxmC7aioSghAoxTOrdnD+1rcXM1btoPTiJCcvSCOSFnHhHBV8kyEWGh/HEDdWZ2rcF\nVUsVpv+45fz+g8VsVzM2yUIFXyREVC9dlC96XsfAW+JZvGUfbQYl8dH8LWrGJv+hgi8SQsLDjN83\nr8L0pxJpUKk4f5mYwl0j57Mx/bDX0SQA+K3gm9n1ZjbbzEaY2fX++lyR/KhiiUJ89HAT/tm1Hut2\nHqL90NkM/yFVzdjyuWwVfDMbZWa7zWxVlvF2ZrbOzFLNbMBF3sYBh4ECQNrlxRWR7DIz7mhUkVl/\naMWNtUrz2rR13Dp8Lim/6Hbn/Cpbt2WaWSJni/VHzrm6vrFwYD3QmrMFfDFwNxAOvJzlLR4G9jjn\nzphZGWCQc+7ei32ubssUyT3frtzBnyemsP/oSXq2qsqTN9agQGS417EkD5zvtsyI7Hyzcy7ZzCpn\nGW4CpDrnNvk+YCzQ2Tn3MtDxAm+3HzjvjcJm1gPoARAXF5edeCKSDe0TynJttZL8fcoahn2/kWmr\ndvJql3o0qlzC62jiJznZwy8PbMt0nOYbOyczu93M3gE+Bt463+uccyOdc42cc41iY2NzEE9Esrqi\nUBSv31mfDx9uwvFTZ7jjnfkMnJTCkRNqxpYf+O2irXNugnPuMefcXc65Hy70WrVWEMlbrWrGMr1f\nIg80q8SH87fQZnAyyevTvY4leSwnBX87UDHTcQXfmIgEgSLREbzQuS5fPHYt0ZFhPDBqEX/8Yjm/\nHj3pdTTJIzkp+IuBGmZWxcyigG7ApNwI5Zyb7JzrERMTkxtvJyIX0KhyCab2ackTN1Tjqx+3c9Og\nZL5ducPrWJIHsntb5hhgPlDLzNLMrLtzLgPoDUwH1gDjnHMpuRFKWzoi/lUgMpw/ta3NpN7NKVMs\nmsc/Xcbjnyxl96HjXkeTXKRumSLyX06dPsO7szcxZNYGCkaG8/zNV9G1YQXMzOtokk1B1S1TK3wR\n70SGh9Hr+up827clNcsU4U9fruCBUYvYtu+o19Ekh7TCF5HzOnPG8enCrbzy7Voc8HTbWjxwbWXC\nwrTaD2RBtcIXkcAQFmbcf21lpvdLpHHlEgycvJo73plP6u5DXkeTyxCQBV9bOiKBpULxQox+qDGD\n7qzPxvTDdBg6h2Hfp3JKzdiCirZ0ROSSpB86wcBJKXyzcgdXlS3GP7vWo2553UIdSLSlIyK5IrZo\nNMPubcCI+xqy5/AJOg+byyvfruX4qdNeR5OLCMiCry0dkcDXru6VzOrXii4NyjMiaSMdhs5m0eZ9\nXseSC9CWjojk2JwNexgwYQVp+49xf7NKPNO+NkWis9WMV/KAtnREJM+0qFGKGf0Sebh5FT5ZuJU2\ng5L4ft1ur2NJFgFZ8LWlIxJ8CkVF8Jdb4vmy53UUio7goQ8W0//zn9h/RM3YAoW2dEQk153IOM2w\n71IZ/sNGrigUyQud6tIh4Uq1Z/ATbemIiN9ER4TTv00tJvVuQdmYgjzx2TIe+3gpuw6qGZuXVPBF\nJM/ElyvGV72u49n2tUlan85Ng5L4fPHPBPLOQihTwReRPBURHsZjraox7alEripbjGfGr+S+9xfy\n8141Y/M3FXwR8YsqpQoz9tFmvHhrXZZvO0DbIcm8P2czp89ote8vAVnwdZeOSGgKCzPua1aJGf0S\naVa1BH+fspquI+axYZeasfmD7tIREU8455j40y+8MDmFwycyePLGGvRsVY2oiIBchwYV3aUjIgHF\nzLj1mvLM7N+KdnXLMmjmejq9NYfl2371OlrIUsEXEU+VKhLNm3dfw7sPNGL/0ZPcNnwuL09dw7GT\nasaW21TwRSQgtI4vw4x+rbircUXeSd5E+6HJLNi01+tYIcVvBd/MwszsJTN708we9NfnikjwiCkY\nycu31+OzR5pyxkG3kQt47quVHDp+yutoISFbBd/MRpnZbjNblWW8nZmtM7NUMxtwkbfpDFQATgFp\nlxdXRPKD66qXYvpTiTzSogpjFv1Mm8HJfLd2l9exgl52V/ijgXaZB8wsHBgGtAfigbvNLN7MEsxs\nSpZHaaAWMM851x94PPemICKhqGBUOM93jGf849dRtEAED49eQt+xP7L38AmvowWtbDWsds4lm1nl\nLMNNgFTn3CYAMxsLdHbOvQx0zPoeZpYG/NY2T1djRCRbrokrzpQnWzLs+1SG/5DK7A17GNipDrfU\nK6tmbJcoJ3v45YFtmY7TfGPnMwFoa2ZvAsnne5GZ9TCzJWa2JD09PQfxRCRUREWE0a91TSY/2YKK\nxQvSZ8yPPPrREnYeUDO2S+G3i7bOuaPOue7OuSedc8Mu8LqRwAvAsqioKH/FE5EgUPvKYkzo1Zzn\nb76KOal7aD0oiTGL1Iwtu3JS8LcDFTMdV/CN5ZhzbrJzrkdMTExuvJ2IhJDwMOORllWZ/lQidcvH\n8OyEldzz7kK27DnidbSAl5OCvxioYWZVzCwK6AZMyo1Q6qUjIhdTqWRhPnu0KS/fnsCq7QdoNzSZ\nd5M3qRnbBWT3tswxwHyglpmlmVl351wG0BuYDqwBxjnnUvIuqojIfzMz7m4Sx8z+rWhRvRQvTV3D\n7cPnsm6nmrGdi5qniUhIcM4xecUOBk5K4dDxU/S6vjpP3FA9XzZjC6rmadrSEZFLZWZ0ql+OWf1b\ncXNCWYb+ewMd35zNT2rG9h9a4YtISPpu7S6e+2oVuw4e5+HmVfhDm1oUjAr3OpZfBNUKX0Qkp26s\nXYYZ/RK5u0kc783ZTNshycxL3eN1LE8FZMHXlo6I5IaiBSJ56bYExvZoRpjBPe8tZMD4FRw4lj+b\nsWlLR0TyhWMnTzNk1nrenb2J2KLRvHhrAq3jy3gdK09oS0dE8rWCUeE82+Eqvn6iOcULRfHoR0vo\n/dky9uSjZmwBWfC1pSMieaVehSuY1LsF/VvXZHrKTloPSuLrH7fni/YM2tIRkXxrw65DPD1+BT/+\n/Cs31i7Ni7fWpdwVBb2OlWPa0hERyaJGmaJ82fM6/tIxnvkb99JmcDKfLNjKmRBtz6CCLyL5WniY\n8XCLKkx/KpH6FWN4/utVdHt3AZtDsBlbQBZ87eGLiL/FlSzEJ92b8lqXeqzZcZB2Q5IZkbSRjNNn\nvI6Wa7SHLyKSxa6Dx3n+61XMXL2LhPIxvNqlHvHlinkdK9u0hy8ikk1lihVg5P0NGXZPA3YcOEan\nt+bw+ox1nMgI7n+dVQVfROQczIyb65VlZr9WdLq6HG9+l8rNb8xh6db9Xke7bCr4IiIXULxwFIPu\nvJoPHmrM0RMZdB0xjxcmp3DkRIbX0S5ZQBZ8XbQVkUBzQ63SzOjfivubVeKDuVtoOySZ2RvSvY51\nSXTRVkTkEi3avI9nxq9g854j3NmoAs91iCemUKTXsf5DF21FRHJJkyol+LZvSx6/vhrjl23npsFJ\nTFu10+tYF6WCLyJyGQpEhvNMu9pMfKI5sUWi6fnJUp74dBnphwK3GZsKvohIDtQtH8PE3s35U9ta\nzFy9i5sGJTF+aVpANmNTwRcRyaHI8DCeuKE6U/u2pHrpIvzhi+U8+MFi0vYf9Traf/FbwTezlmY2\nwszeM7N5/vpcERF/qV66CF88di0Db4lnyZZ9tB2czEfztwRMM7ZsFXwzG2Vmu81sVZbxdma2zsxS\nzWzAhd7DOTfbOdcTmAJ8ePmRRUQCV1iY8fvmZ5uxNahUnL9MTOGukfPZmH7Y62jZXuGPBtplHjCz\ncGAY0B6IB+42s3gzSzCzKVkepTN96z3AZ7mQXUQkYFUsUYiPHm7Cv+6oz/pdh2k/dDbDf0jllIfN\n2CKy8yLnXLKZVc4y3ARIdaFdQEcAAAaMSURBVM5tAjCzsUBn59zLQMdzvY+ZxQEHnHOHzvdZZtYD\n6AEQFxeXnXgiIgHJzOjasAKJNUvx14kpvDZtHd+s2MGrXepRt3yM3/PkZA+/PLAt03Gab+xCugMf\nXOgFzrmRwAvAsqioqBzEExEJDKWLFuDt+xry9r0N2HXwBJ2HzeW1aWs5fsq/zdj8epeOc+6vzrmL\nXrB1zk12zvWIifH/74AiInmlfUJZZvVP5LZryjP8h410eGM2S7bs89vn56TgbwcqZjqu4BvLMfXS\nEZFQdUWhKP51R30+ergJJ06d4Y535vPXias47IdmbDkp+IuBGmZWxcyigG7ApNwIpRW+iIS6xJqx\nzOiXyIPXVuajBVtpOziZpPV524wtu7dljgHmA7XMLM3MujvnMoDewHRgDTDOOZeSG6G0wheR/KBw\ndAQDO9Xhi8eupUBkGA+OWsQfxi3n16Mn8+Tz1C1TRCQAHD91mre+S+XtpI0ULxTFO/c3oGGlEpf1\nXkHVLVMrfBHJbwpEhvPHtrWY1Ls5V5UtSqWShXP9M7TCFxEJMVrhi4jkcwFZ8HWXjohI7gvIgi8i\nIrkvIAu+tnRERHJfQBZ8bemIiOS+gCz4IiKS+1TwRUTyiYAs+NrDFxHJfQH9F6/MLB3YepnfXgrY\nk4txvKS5BJ5QmQdoLoEqJ3Op5JyLzToY0AU/J8xsybn+plkw0lwCT6jMAzSXQJUXcwnILR0REcl9\nKvgiIvlEKBf8kV4HyEWaS+AJlXmA5hKocn0uIbuHLyIi/y2UV/giIpKJCr6ISD4RkgXfzNqZ2Toz\nSzWzAV7nuRgz22JmK83sJzNb4hsrYWYzzWyD79fivnEzszd8c1thZg08zj7KzHab2apMY5ec3cwe\n9L1+g5k9GEBzGWhm233n5icz65Dpa8/65rLOzNpmGvf058/MKprZ92a22sxSzKyvbzzozssF5hKM\n56WAmS0ys+W+ubzgG69iZgt9uT43syjfeLTvONX39coXm+NFOedC6gGEAxuBqkAUsByI9zrXRTJv\nAUplGXsNGOB7PgB41fe8A/AtYEAzYKHH2ROBBsCqy80OlAA2+X4t7ntePEDmMhD44zleG+/72YoG\nqvh+5sID4ecPKAs08D0vCqz35Q2683KBuQTjeTGgiO95JLDQ9997HNDNNz4CeNz3vBcwwve8G/D5\nheaYnQyhuMJvAqQ65zY5504CY4HOHme6HJ2BD33PPwRuzTT+kTtrAXCFmZX1IiCAcy4Z2Jdl+FKz\ntwVmOuf2Oef2AzOBdnmf/r+dZy7n0xkY65w74ZzbDKRy9mfP858/59wO59wy3/NDwBqgPEF4Xi4w\nl/MJ5PPinHOHfYeRvocDbgS+9I1nPS+/na8vgd+ZmXH+OV5UKBb88sC2TMdpXPgHJBA4YIaZLTWz\nHr6xMs65Hb7nO4EyvufBML9LzR7oc+rt2+oY9ds2CEEyF982wDWcXU0G9XnJMhcIwvNiZuFm9hOw\nm7O/gW4EfnXOZZwj138y+75+AChJDuYSigU/GLVwzjUA2gNPmFli5i+6s3+OC8r7Z4M5u8/bQDXg\namAH8Lq3cbLPzIoA44GnnHMHM38t2M7LOeYSlOfFOXfaOXc1UIGzq/La/vz8UCz424GKmY4r+MYC\nlnNuu+/X3cBXnP1B2PXbVo3v192+lwfD/C41e8DOyTm3y/c/6RngXf7/H50Dei5mFsnZAvmpc26C\nbzgoz8u55hKs5+U3zrlfge+Bazm7hRZxjlz/yez7egywlxzMJRQL/mKghu/KdxRnL3ZM8jjTeZlZ\nYTMr+ttzoA2wirOZf7sr4kFgou/5JOAB350VzYADmf6YHiguNft0oI2ZFff90byNb8xzWa6P3MbZ\ncwNn59LNdydFFaAGsIgA+Pnz7fO+D6xxzg3K9KWgOy/nm0uQnpdYM7vC97wg0Jqz1yS+B7r6Xpb1\nvPx2vroC3/n+ZHa+OV6cP69S++vB2bsO1nN2f+w5r/NcJGtVzl5xXw6k/JaXs3t1/wY2ALOAEu7/\nX+kf5pvbSqCRx/nHcPaP1Kc4u5fY/XKyAw9z9uJTKvBQAM3lY1/WFb7/0cpmev1zvrmsA9oHys8f\n0IKz2zUrgJ98jw7BeF4uMJdgPC/1gB99mVcBf/GNV+VswU4FvgCifeMFfMepvq9XvdgcL/ZQawUR\nkXwiFLd0RETkHFTwRUTyCRV8EZF8QgVfRCSfUMEXEcknVPBFRPIJFXwRkXzi/wGLWcU8yZ6BKwAA\nAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(recoder['global_step'],recoder['lr'])\n",
    "plt.yscale('log')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "xAHSnRWxV25i"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "name": "Regulization_lr_decay_dropout.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "072302059898401c804ee7034832f9f9": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_270a2223fc304ab1bbe6570299971020",
      "placeholder": "​",
      "style": "IPY_MODEL_45c56abd3b4340c6973365a8505a1489",
      "value": "9920512it [00:30, 1035937.44it/s]"
     }
    },
    "09f1267e88274369a38f089ae15ece1e": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "131cbda720564706ba6e8e65ab852dd0": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_a5bf223930554049acfbec1722530722",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_3b38db19f8a74ff3932b1326cb7793a1",
      "value": 1
     }
    },
    "15a7be22a6d2460a9378feefdc631f4d": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "1720d7f79cf44c86957f956e3d001b92": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "270a2223fc304ab1bbe6570299971020": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "31f3098f51774b12a29bd4457b9a627c": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_15a7be22a6d2460a9378feefdc631f4d",
      "placeholder": "​",
      "style": "IPY_MODEL_eead47fd96a24e16ab296a71fa19e50e",
      "value": "32768it [00:00, 43988.83it/s]"
     }
    },
    "3b38db19f8a74ff3932b1326cb7793a1": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "45c56abd3b4340c6973365a8505a1489": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "54175c440eaf41618b6c220ebdac6953": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_131cbda720564706ba6e8e65ab852dd0",
       "IPY_MODEL_d369f8986df94b5a815bb143fe2bae1b"
      ],
      "layout": "IPY_MODEL_b951953cae084fbb903835217eadc1d7"
     }
    },
    "55c8e2d421fe41f7bfd17873af455104": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "5ac3d619429e4705b3602264f0efabaf": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "6a5c3b4996f44a9a886ae2a749991633": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "6f36fb23c8bf4b4e885961de455303fa": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "703b6d9d7cc4475a8156891551b3cb3d": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "91f55b603d684df2a5cc6105edc095a8": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_c085442b5b9843edbb7846eb85f9dcd0",
      "placeholder": "​",
      "style": "IPY_MODEL_55c8e2d421fe41f7bfd17873af455104",
      "value": "1654784it [00:01, 865538.56it/s]"
     }
    },
    "936cbfbcb00242bfb9687aa41e5da83e": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_703b6d9d7cc4475a8156891551b3cb3d",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_5ac3d619429e4705b3602264f0efabaf",
      "value": 1
     }
    },
    "96464aca53164b549ba46339054ee2e3": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "9ab389cdb3664dc9a5e5d73a131de3cf": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "info",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_9dadb04364204ba580546c5ed46b533d",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_ab631f0323c44fd28bca2122e78972fd",
      "value": 1
     }
    },
    "9dadb04364204ba580546c5ed46b533d": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "a5bf223930554049acfbec1722530722": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "ab631f0323c44fd28bca2122e78972fd": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "b951953cae084fbb903835217eadc1d7": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "c085442b5b9843edbb7846eb85f9dcd0": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "c0e637c4e34941a596aed9045e00d800": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "c2489720a216407c86df1dac212e6860": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_c293b4df711c4728b743a9d81f9a8d92",
       "IPY_MODEL_31f3098f51774b12a29bd4457b9a627c"
      ],
      "layout": "IPY_MODEL_6a5c3b4996f44a9a886ae2a749991633"
     }
    },
    "c293b4df711c4728b743a9d81f9a8d92": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_1720d7f79cf44c86957f956e3d001b92",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_09f1267e88274369a38f089ae15ece1e",
      "value": 1
     }
    },
    "cd2b2f1bea8c44d1be2c8b1ed96e6b1d": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "cd3348b28a3d4c52927785ef57be38b5": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_936cbfbcb00242bfb9687aa41e5da83e",
       "IPY_MODEL_91f55b603d684df2a5cc6105edc095a8"
      ],
      "layout": "IPY_MODEL_96464aca53164b549ba46339054ee2e3"
     }
    },
    "d1228d0da4fe431ab876e1767f93486b": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_9ab389cdb3664dc9a5e5d73a131de3cf",
       "IPY_MODEL_072302059898401c804ee7034832f9f9"
      ],
      "layout": "IPY_MODEL_cd2b2f1bea8c44d1be2c8b1ed96e6b1d"
     }
    },
    "d369f8986df94b5a815bb143fe2bae1b": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_c0e637c4e34941a596aed9045e00d800",
      "placeholder": "​",
      "style": "IPY_MODEL_6f36fb23c8bf4b4e885961de455303fa",
      "value": "8192it [00:00, 16215.02it/s]"
     }
    },
    "eead47fd96a24e16ab296a71fa19e50e": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
